{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "551e3f8e-3058-4ff0-8007-a01973648e4b",
   "metadata": {},
   "source": [
    "# Pandas快速入门与实践"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ca3225f-9e95-494c-b771-937e3aac17fe",
   "metadata": {},
   "source": [
    "## Series基本操作"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c7d24ad-aec5-49f4-bd38-9cc2ed48ddbe",
   "metadata": {},
   "source": [
    "### 数据选择与访问"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7edcfd0c-7f26-4d32-94a3-90a20b019c84",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#np.random.seed(96)  # 固定随机种子\n",
    "#v = np.random.randint(1, 10, size=10)   # 生成于1到10之间的10个随机正整数\n",
    "#print(v)\n",
    "\n",
    "s = pd.Series([5, 7, 5, 2, 9, np.nan, 3, 7, 7, np.nan, 3, 8])\n",
    "print(s)   # 打印整个Series，元素多的时候慎用\n",
    "print(s.head())  # 显示序列的前5个元素\n",
    "print(s.tail(3))  # 显示序列的后3个元素\n",
    "print(s[1])  # 打印索引1的元素值\n",
    "print(s.iloc[1])  # 打印索引1的元素值\n",
    "print(s[[1, 11]])  # 打印索引1和11元素值\n",
    "print(s[1:5])  # 打印索引1到5之间的元素值，不包括索引5\n",
    "print(s[s>7])  # 打印元素值打7的元素"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65a4b1e7-0e96-4e33-a119-65159a29961f",
   "metadata": {},
   "source": [
    "### 数据清洗操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7e6d6cb-a4ce-4d25-934b-e86bfbd66c2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "s = pd.Series([5, 7, 5, 2, 9, np.nan, 3, 7, 7, np.nan, 3, 8])\n",
    "print(s.info())  # 打印Series信息\n",
    "\n",
    "print(s.isna())  # 返回一个布尔型Series，标记每个元素是否为NaN\n",
    "\n",
    "print(s.dropna())  # 删除缺失值，返回视图（不对原Series进行操作）\n",
    "print(s.fillna(10))  # 用指定的值填充\n",
    "print(s.interpolate())  # 插值填充，返回视图\n",
    "\n",
    "print(s.duplicated())   # 检查重复值\n",
    "print(s.drop_duplicates())  # 移除重复值，每个元素值仅保留一份\n",
    "s.interpolate().drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "740024c2-b6b2-48b5-97eb-526f23265e00",
   "metadata": {},
   "source": [
    "### 统计与聚合操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b7e53a0-ccb4-47b7-be3b-ea17277fe136",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "s = pd.Series([5, 7, 5, 2, 9, np.nan, 3, 7, 7, np.nan, 3, 8])\n",
    "print(s.sum())   # 求和\n",
    "print(s.dropna().sum())  # 非缺失值求和\n",
    "print(s.dropna().median())  # 非缺失值求中位数\n",
    "print(s.dropna().quantile())  # 求四分之一分位数\n",
    "print(s.dropna().unique())  # 返回不包含NaN的惟一值元素列表\n",
    "\n",
    "s_dates = pd.Series([\"2022-12-1\", \"2023-07-21\", \"2024-06-01\", \"2025-07-19\"])   # 生成日期Series\n",
    "print(s_dates)\n",
    "s_dates = pd.to_datetime(s_dates)   # 转换为Pandas的日期格式\n",
    "print(s_dates)\n",
    "\n",
    "print(s_dates.dt.year)   # 打印年份\n",
    "print(s_dates.dt.month) # 打印月份\n",
    "print(s_dates.dt.day)   # 打印Day of Month\n",
    "print(s_dates.dt.weekday)  # 打印Day of Week"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bae8b42-f0a9-4847-afaa-6176d706d0b2",
   "metadata": {},
   "source": [
    "### Series绘图操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43e18832-ac4e-4b10-9fbc-01e89c331254",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "s = pd.Series([5, 7, 5, 2, 9, np.nan, 3, 7, 7, np.nan, 3, 8])\n",
    "\n",
    "fig = s.dropna().plot(kind=\"line\")\n",
    "plt.title(\"Series序列非空元素值\")\n",
    "plt.xlabel(\"索引\")\n",
    "plt.ylabel(\"元素值\")\n",
    "plt.show()\n",
    "\n",
    "fig = s.dropna().plot(kind=\"bar\")\n",
    "plt.title(\"Series序列非空元素值\")\n",
    "plt.xlabel(\"索引\")\n",
    "plt.ylabel(\"元素值\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84713e29-98d6-40e1-8125-fdfe034669db",
   "metadata": {},
   "source": [
    "### 应用函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9aa12c3-0c21-4a5f-b2d7-db46d5fafc47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建 Series\n",
    "s = pd.Series(['cat', 'dog', 'bird', 'tigger'])\n",
    "\n",
    "# 定义映射字典\n",
    "mapping = {'cat': '猫', 'dog': '狗', 'bird': '鸟'}\n",
    "\n",
    "# 使用map()\n",
    "print(f\"使用map()进行字典映射：\\n{s.map(mapping)}\")\n",
    "# 使用map()，忽略缺失值\n",
    "print(f\"使用map()进行字典映射：\\n{s.map(mapping, na_action='ignore')}\")\n",
    "\n",
    "# 创建第二个 Series\n",
    "s2 = pd.Series([1, 2, None, 3, 4])\n",
    "\n",
    "# 使用apply()映射\n",
    "result_apply = s2.apply(lambda x: x ** 2)\n",
    "print(f\"使用apply()应用函数：\\n{result_apply}\")\n",
    "\n",
    "# 使用apply()映射，模拟忽略缺失值\n",
    "result_apply = s2.apply(lambda x: x * 2 if pd.notnull(x) else x)\n",
    "print(f\"使用apply()应用函数：\\n{result_apply}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36e9a087-3544-42b7-8921-1dae005cb010",
   "metadata": {},
   "source": [
    "## DataFrame实践"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4be965db-5c83-46c4-b3a2-3a1ffb9fb602",
   "metadata": {},
   "source": [
    "### 基于数据集创建DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbf1cf65-6730-4266-aafd-94cc79c1d6e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_iris\n",
    "import seaborn as sns\n",
    "\n",
    "# 加载 tips 数据集\n",
    "tips = sns.load_dataset('tips')\n",
    "df = pd.DataFrame(tips)  # 确保是 Pandas DataFrame\n",
    "print(f\"Seaborn的tip数据集预览：\\n {df.head()}\")\n",
    "\n",
    "# 加载Iris数据集\n",
    "iris = load_iris()\n",
    "\n",
    "# 创建DataFrame，未指定列名称\n",
    "df = pd.DataFrame(data=iris.data)\n",
    "print(f\"\\nScikit-Learn的iris数据集预览：\\n {df.head()}\")\n",
    "\n",
    "# 创建DataFrame，指定列名称为Feature名称\n",
    "df = pd.DataFrame(data=iris.data, columns=iris.feature_names)\n",
    "df['target'] = iris.target  # 添加目标列\n",
    "df['target'] = df['target'].map({0: 'setosa', 1: 'versicolor', 2: 'virginica'})  # 将目标值映射为类别名称\n",
    "\n",
    "# 打印 DataFrame 前几行以确认结构\n",
    "print(f\"\\nScikit-Learn的iris数据集预览(使用Feature Name作为列名称，并添加了target字段)：\\n {df.head()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de5fd67-80be-443c-99d6-4aa20615bde3",
   "metadata": {},
   "source": [
    "### 数据的基本操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "683e5138-c7ca-4be4-a2b6-a9d1296313ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "# 加载 tips 数据集\n",
    "tips = sns.load_dataset('tips')\n",
    "df = pd.DataFrame(tips)\n",
    "\n",
    "print(f\"后5行数据：\\n{df.tail()}\")\n",
    "print(f\"\\n索引信息：\\n{df.index}\")\n",
    "print(f\"\\n列名信息：\\n{df.columns}\")\n",
    "print(f\"\\n底层的NumPy数组（前3行）：\\n{df.head(3).values}\")\n",
    "# print(f\"\\n底层的NumPy数组（前3行）：\\n{df.values[:3]}\")\n",
    "print(f\"\\n各列数据类型：\\n{df.dtypes}\")\n",
    "print(f\"\\nDataFrame表格的形状：\\n{df.shape}\")\n",
    "\n",
    "print(\"\\nDataFrame的数据信息:\")\n",
    "print(df.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cef02e89-ddd2-4fc9-8099-11d917cb5992",
   "metadata": {},
   "source": [
    "### 聚合操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f0af7ac-1c6a-4c05-8026-73fc48a5cfdd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame([[1,2,3,4],[5,6,7,8],[10,20,30,40]], index=['row0', 'row1', 'row2'], columns=['col0', 'col1', 'col2', 'col3'])\n",
    "print(df.head())\n",
    "\n",
    "# 对所有行或所有列求和：df.sum(axis=0|axis=1)\n",
    "print(f\"对列求和（以行为轴）:\\n{df.sum(axis=0)}\")\n",
    "print(f\"对行求和（以列为轴）:\\n{df.sum(axis=1)}\")\n",
    "\n",
    "# 对指定列求和：df['col_name'].sum()\n",
    "print(f\"对指定的列求和：\\n{df['col0'].sum()}\")\n",
    "\n",
    "# 对指定的行（使用行标签）求和：df.loc['row_label'].sum()\n",
    "print(f\"对指定的行求和：\\n{df.loc['row0'].sum()}\")\n",
    "\n",
    "# 对指定的行（使用行索引）求和：df.iloc[index].sum()\n",
    "print(f\"对指定的行求和：\\n{df.iloc[1].sum()}\")\n",
    "\n",
    "print(\"对指定范围内的多行求和\")\n",
    "print(df.iloc[[0, 1]].sum(axis=1))\n",
    "\n",
    "print(\"对指定范围内的多行求和之后，再求总和\")\n",
    "print(df.iloc[[0, 1]].sum(axis=1).sum())\n",
    "\n",
    "# 求所有数值列之间的相关系数\n",
    "print(\"\\n所有数值列之间的相关系数:\")\n",
    "print(df.corr())\n",
    "\n",
    "# 指定的两个列之间的相关系数\n",
    "print(\"\\n指定的两个数值列之间的相关系数:\")\n",
    "print(df['col1'].corr(df['col2']))\n",
    "\n",
    "# 求所有数值列之间的协方差\n",
    "print(\"\\n所有数值列之间的协方差:\")\n",
    "print(df.cov())\n",
    "\n",
    "# 指定的两个列之间的协方差\n",
    "print(\"\\n指定的两个数值列之间的协方差:\")\n",
    "print(df['col1'].cov(df['col2']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e562edf9-302a-422d-9202-17b374b5b91c",
   "metadata": {},
   "source": [
    "### 相关系数矩阵和协方差矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da4d500a-ed3a-43d2-bb35-3ea93dfec73d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 加载 tips 数据集\n",
    "tips = sns.load_dataset('tips')\n",
    "df = pd.DataFrame(tips)  # 转换为 Pandas DataFrame\n",
    "\n",
    "# 选择数值列\n",
    "numeric_cols = ['total_bill', 'tip', 'size']\n",
    "df_numeric = df[numeric_cols]\n",
    "\n",
    "# 计算相关系数矩阵（Pearson）\n",
    "print(\"\\n=== 相关系数矩阵（Pearson） ===\")\n",
    "corr_matrix = df_numeric.corr(method='pearson')\n",
    "print(corr_matrix)\n",
    "\n",
    "# 计算协方差矩阵\n",
    "print(\"\\n=== 协方差矩阵 ===\")\n",
    "cov_matrix = df_numeric.cov()\n",
    "print(cov_matrix)\n",
    "\n",
    "# 可视化相关系数矩阵\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', center=0, vmin=-1, vmax=1)\n",
    "plt.title('Tips数据集上数值列的相关系数矩阵(Numeric Columns)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca00c6b6-2e5d-4003-920f-735589fac364",
   "metadata": {},
   "source": [
    "### 分组聚合功能示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "513e4089-e4d5-4d55-8185-472ed45bc7ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 加载 tips 数据集\n",
    "tips = sns.load_dataset('tips')\n",
    "df = pd.DataFrame(tips)\n",
    "\n",
    "# 单列分组：按 day 分组\n",
    "print(\"\\n=== 按星期分组的平均账单和小费 ===\")\n",
    "day_group = df.groupby('day')[['total_bill', 'tip']].mean()\n",
    "print(day_group)\n",
    "\n",
    "# 多个聚合函数\n",
    "print(\"\\n=== 按星期分组的多种统计（均值、总和、计数） ===\")\n",
    "day_agg = df.groupby('day')[['total_bill', 'tip']].agg(['mean', 'sum', 'count'])\n",
    "print(day_agg)\n",
    "\n",
    "# 多列分组：按 day 和 time 分组\n",
    "print(\"\\n=== 按星期和就餐时间分组的平均账单和小费 ===\")\n",
    "day_time_group = df.groupby(['day', 'time'])[['total_bill', 'tip']].mean()\n",
    "print(day_time_group)\n",
    "\n",
    "# 自定义聚合函数：计算小费比例均值\n",
    "print(\"\\n=== 按星期分组的小费比例均值 ===\")\n",
    "def tip_ratio_mean(group):\n",
    "    return (group['tip'] / group['total_bill']).mean()\n",
    "tip_ratio = df.groupby('day').apply(tip_ratio_mean)\n",
    "print(tip_ratio)\n",
    "\n",
    "# 可视化\n",
    "print(\"\\n=== 可视化：按星期分组的平均账单和小费 ===\")\n",
    "day_group.plot(kind='bar', figsize=(10, 6))\n",
    "plt.title('Average Total Bill and Tip by Day')\n",
    "plt.xlabel('Day')\n",
    "plt.ylabel('Amount ($)')\n",
    "plt.legend(['Total Bill', 'Tip'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9ea8156-26ac-4410-9293-dabc15e0d96f",
   "metadata": {},
   "source": [
    "### 数据清洗示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8180bb16-8ba4-4c23-a3c3-de1dabcb30eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_iris\n",
    "import numpy as np\n",
    "\n",
    "# 加载 iris 数据集\n",
    "iris = load_iris()\n",
    "df = pd.DataFrame(data=iris.data, columns=iris.feature_names)\n",
    "df['species'] = iris.target_names[iris.target]  # 添加种类列\n",
    "\n",
    "# 查看基本信息\n",
    "print(\"=== 数据集基本信息 ===\")\n",
    "print(df.info())\n",
    "print(\"\\n前 5 行数据：\")\n",
    "print(df.head())\n",
    "\n",
    "# 创建脏数据副本\n",
    "df_dirty = df.copy()\n",
    "\n",
    "# 1. 随机引入缺失值\n",
    "np.random.seed(42)\n",
    "mask = np.random.choice([True, False], size=df.shape, p=[0.1, 0.9])  # 10% 概率缺失\n",
    "df_dirty[mask] = np.nan\n",
    "\n",
    "# 2. 添加重复行\n",
    "df_duplicate = pd.concat([df_dirty, df_dirty.iloc[:5]], ignore_index=True)\n",
    "\n",
    "# 3. 引入异常值（将部分 sepal length 设置为负值或极值）\n",
    "df_dirty.loc[0:2, 'sepal length (cm)'] = -1.0  # 负值异常\n",
    "df_dirty.loc[3, 'sepal length (cm)'] = 100.0   # 极值异常\n",
    "\n",
    "# 4. 引入不一致的种类名称\n",
    "df_dirty.loc[5:7, 'species'] = 'SETOSA'  # 大写不一致\n",
    "\n",
    "# 查看脏数据\n",
    "print(\"\\n=== 脏数据前 10 行 ===\")\n",
    "print(df_dirty.head(10))\n",
    "print(\"\\n脏数据信息：\")\n",
    "print(df_dirty.info())\n",
    "\n",
    "# 检查缺失值\n",
    "print(\"\\n=== 缺失值统计 ===\")\n",
    "print(df_dirty.isnull().sum())\n",
    "\n",
    "# 填充数值列的缺失值（使用均值）\n",
    "df_cleaned = df_dirty.copy()\n",
    "numeric_cols = ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']\n",
    "for col in numeric_cols:\n",
    "    df_cleaned[col] = df_cleaned[col].fillna(df_cleaned[col].mean())\n",
    "\n",
    "# 填充分类列的缺失值（使用众数）\n",
    "df_cleaned['species'] = df_cleaned['species'].fillna(df_cleaned['species'].mode()[0])\n",
    "\n",
    "print(\"\\n=== 填充后缺失值统计 ===\")\n",
    "print(df_cleaned.isnull().sum())\n",
    "\n",
    "# 检查重复行（基于 df_duplicate）\n",
    "print(\"\\n=== 重复行数量 ===\")\n",
    "print(df_duplicate.duplicated().sum())\n",
    "\n",
    "# 删除重复行\n",
    "df_duplicate_cleaned = df_duplicate.drop_duplicates()\n",
    "print(\"\\n=== 删除重复行后的形状 ===\")\n",
    "print(df_duplicate_cleaned.shape)\n",
    "\n",
    "# 检测负值异常\n",
    "print(\"\\n=== 负值异常（sepal length） ===\")\n",
    "print(df_cleaned[df_cleaned['sepal length (cm)'] < 0])\n",
    "\n",
    "# 替换负值为均值\n",
    "mean_sepal_length = df_cleaned['sepal length (cm)'][df_cleaned['sepal length (cm)'] > 0].mean()\n",
    "df_cleaned.loc[df_cleaned['sepal length (cm)'] < 0, 'sepal length (cm)'] = mean_sepal_length\n",
    "\n",
    "# 检测极值（例如，sepal length > 10）\n",
    "print(\"\\n=== 极值异常（sepal length） ===\")\n",
    "print(df_cleaned[df_cleaned['sepal length (cm)'] > 10])\n",
    "\n",
    "# 替换极值为均值\n",
    "df_cleaned.loc[df_cleaned['sepal length (cm)'] > 10, 'sepal length (cm)'] = mean_sepal_length\n",
    "\n",
    "print(\"\\n=== 异常值处理后前 5 行 ===\")\n",
    "print(df_cleaned.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d963d40-e153-4145-887a-f26d241268fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "\n",
    "# 加载iris数据集\n",
    "# 使用scikit-learn库加载经典的iris(鸢尾花) 数据集。\n",
    "# iris数据集包含4个特征 (萼片长度、萼片宽度、花瓣长度、花瓣宽度) 和3种鸢尾花类别。\n",
    "iris = load_iris()\n",
    "# 将 iris 数据转换为 Pandas DataFrame。\n",
    "# data=iris.data 包含了数值特征。\n",
    "# columns=iris.feature_names 设置了 DataFrame 的列名。\n",
    "df = pd.DataFrame(data=iris.data, columns=iris.feature_names)\n",
    "# 将鸢尾花的类别信息 (target) 添加为新列 'species'。\n",
    "# iris.target 是数值型类别 (0, 1, 2)，iris.target_names 提供了对应的名称 ('setosa', 'versicolor', 'virginica')。\n",
    "df['species'] = iris.target_names[iris.target]\n",
    "\n",
    "# 创建脏数据\n",
    "# 复制原始 DataFrame，以便在上面制造“脏数据”而不影响原始数据。\n",
    "df_dirty = df.copy()\n",
    "# 设置随机种子，确保每次运行代码时生成的随机数相同，从而使脏数据生成过程可复现。\n",
    "np.random.seed(42)\n",
    "# 随机生成一个布尔掩码，用于在 DataFrame 中随机选择一些位置。\n",
    "# np.random.choice([True, False], size=df.shape, p=[0.1, 0.9]) 表示有10%的概率为True，90%的概率为False。\n",
    "# 这样，大约10%的数据点将被选中用于制造缺失值。\n",
    "mask = np.random.choice([True, False], size=df.shape, p=[0.1, 0.9])\n",
    "# 将掩码为True的位置的数据替换为NaN (Not a Number)，即缺失值。\n",
    "df_dirty[mask] = np.nan\n",
    "# 在 'sepal length (cm)' 列的第0到2行 (包含 0、1、2 行) 引入异常的负值 (-1.0)。\n",
    "df_dirty.loc[0:2, 'sepal length (cm)'] = -1.0\n",
    "# 在 'sepal length (cm)' 列的第3行引入异常的过大值 (100.0)。\n",
    "df_dirty.loc[3, 'sepal length (cm)'] = 100.0\n",
    "# 在 'species' 列的第5到7行 (包含5、6、7行) 引入大小写不一致的类别值 ('SETOSA')。\n",
    "df_dirty.loc[5:7, 'species'] = 'SETOSA'\n",
    "# 通过将df_dirty的前5行再次添加到df_dirty的末尾来创建重复行，模拟重复数据。\n",
    "df_duplicate = pd.concat([df_dirty, df_dirty.iloc[:5]], ignore_index=True)\n",
    "\n",
    "# 数据清洗\n",
    "# 复制包含脏数据的 DataFrame，用于进行清洗操作，保留原始脏数据。\n",
    "df_cleaned = df_dirty.copy()\n",
    "\n",
    "# 1. 处理缺失值\n",
    "# 定义数值类型的列名列表，这些列通常需要进行数值填充。\n",
    "numeric_cols = ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']\n",
    "# 遍历数值列，使用该列的均值填充其中的 NaN 值。\n",
    "for col in numeric_cols:\n",
    "    df_cleaned[col] = df_cleaned[col].fillna(df_cleaned[col].mean())\n",
    "# 对于非数值 (类别) 列 'species'，使用众数 (出现次数最多的值) 填充其中的 NaN 值。\n",
    "# .mode()[0] 是因为 mode() 可能返回多个众数，我们取第一个。\n",
    "df_cleaned['species'] = df_cleaned['species'].fillna(df_cleaned['species'].mode()[0])\n",
    "\n",
    "# 2. 处理重复值\n",
    "# 对包含重复数据的DataFrame (df_duplicate) 进行去重操作。\n",
    "# 默认情况下，drop_duplicates()会删除所有重复行，只保留第一次出现的行。\n",
    "df_duplicate_cleaned = df_duplicate.drop_duplicates()\n",
    "\n",
    "# 3. 处理异常值\n",
    "# 计算 'sepal length (cm)' 列中所有大于0的值的均值，用于替换异常值。\n",
    "# 这样可以避免负值或极大的异常值影响均值的计算。\n",
    "mean_sepal_length = df_cleaned['sepal length (cm)'][df_cleaned['sepal length (cm)'] > 0].mean()\n",
    "# 识别并替换 'sepal length (cm)' 列中小于0的异常值，将其替换为之前计算的均值。\n",
    "df_cleaned.loc[df_cleaned['sepal length (cm)'] < 0, 'sepal length (cm)'] = mean_sepal_length\n",
    "# 识别并替换 'sepal length (cm)' 列中大于10的异常值 (例如 100.0)，将其替换为均值。\n",
    "# 这里假设萼片长度通常不会超过10厘米。\n",
    "df_cleaned.loc[df_cleaned['sepal length (cm)'] > 10, 'sepal length (cm)'] = mean_sepal_length\n",
    "\n",
    "# 4. 数据类型转换\n",
    "# 将 'species' 列的数据类型转换为 'category' (类别型)。\n",
    "# 类别型数据类型在存储和处理具有有限数量唯一值的列时更高效，并能提供更好的性能。\n",
    "df_cleaned['species'] = df_cleaned['species'].astype('category')\n",
    "\n",
    "# 5. 值替换\n",
    "# 将 'species' 列中所有值为 'SETOSA' (大写) 的条目替换为 'setosa' (小写)，实现大小写统一。\n",
    "df_cleaned['species'] = df_cleaned['species'].replace('SETOSA', 'setosa')\n",
    "\n",
    "# 查看清洗结果\n",
    "print(\"\\n=== 清洗后数据信息 ===\")\n",
    "# 打印清洗后 DataFrame 的摘要信息，包括每列的非空值数量和数据类型，确认缺失值是否已处理。\n",
    "print(df_cleaned.info())\n",
    "print(\"\\n=== 清洗后前5行 ===\")\n",
    "# 打印清洗后 DataFrame 的前 5 行，以便快速查看数据清洗效果。\n",
    "print(df_cleaned.head())\n",
    "print(\"\\n=== 清洗后重复行数量 ===\")\n",
    "# 打印 df_duplicate_cleaned 中是否存在重复行。期望结果为 0。\n",
    "print(df_duplicate_cleaned.duplicated().sum())\n",
    "print(\"\\n=== 清洗后species唯一值 ===\")\n",
    "# 打印 'species' 列的所有唯一值，确认大小写统一和异常值是否已处理。\n",
    "print(df_cleaned['species'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2fb7c1d7-826a-42ad-a6e4-3c15f3428c68",
   "metadata": {},
   "source": [
    "## DataFrame的高级功能示例"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a465cb54-12bf-4f3b-9473-de2064756054",
   "metadata": {},
   "source": [
    "### DF的合并与连接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b52d9b1-0d25-4da5-ad66-85bc329f3f22",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# 创建两个示例 DataFrame\n",
    "df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],\n",
    "                    'B': ['B0', 'B1', 'B2', 'B3'],\n",
    "                    'C': ['C0', 'C1', 'C2', 'C3'],\n",
    "                    'D': ['D0', 'D1', 'D2', 'D3']},\n",
    "                   index=[0, 1, 2, 3])\n",
    "\n",
    "df2 = pd.DataFrame({'A': ['A4', 'A5', 'A6', 'A7'],\n",
    "                    'B': ['B4', 'B5', 'B6', 'B7'],\n",
    "                    'C': ['C4', 'C5', 'C6', 'C7'],\n",
    "                    'D': ['D4', 'D5', 'D6', 'D7']},\n",
    "                   index=[4, 5, 6, 7])\n",
    "\n",
    "df3 = pd.DataFrame({'F': ['F0', 'F1', 'F2', 'F3'],\n",
    "                    'G': ['G0', 'G1', 'G2', 'G3'],\n",
    "                    'H': ['H0', 'H1', 'H2', 'H3'],\n",
    "                    'I': ['I0', 'I1', 'I2', 'I3']},\n",
    "                   index=[0, 1, 2, 3])\n",
    "\n",
    "print(\"df1:\")\n",
    "print(df1)\n",
    "print(\"\\ndf2:\")\n",
    "print(df2)\n",
    "print(\"\\ndf3:\")\n",
    "print(df3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "022c6605-b766-4cfe-84cb-fb0564078e7b",
   "metadata": {},
   "source": [
    "### 按行连接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3cf17d2-dc6f-467f-9ebc-a9c139d58784",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 沿着行连接 df1 和 df2\n",
    "# 默认 axis=0，表示按行连接\n",
    "result_rows = pd.concat([df1, df2])\n",
    "print(\"\\n--- 按行连接 df1 和 df2 (默认) ---\")\n",
    "print(result_rows)\n",
    "# 此时索引是重复的，因为 df1 的索引是 0,1,2,3，df2 的索引是 4,5,6,7\n",
    "\n",
    "# 忽略原始索引并生成新的连续索引\n",
    "result_rows_ignore_index = pd.concat([df1, df2], ignore_index=True)\n",
    "print(\"\\n--- 按行连接 df1 和 df2 (忽略原始索引) ---\")\n",
    "print(result_rows_ignore_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dae3ac2e-52f7-4236-962d-b4dcf927ebbb",
   "metadata": {},
   "source": [
    "### 按列连接示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1010cdd-f9e7-4b3e-a47a-0b8f808be210",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 沿着列连接 df1 和 df3\n",
    "# df1 和 df3 的行索引相同，列名不同\n",
    "result_cols = pd.concat([df1, df3], axis=1)\n",
    "print(\"\\n--- 按列连接 df1 和 df3 ---\")\n",
    "print(result_cols)\n",
    "\n",
    "# 如果行索引不匹配，会有 NaN\n",
    "df4 = pd.DataFrame({'J': ['J0', 'J1'], 'K': ['K0', 'K1']}, index=[0, 5])\n",
    "print(df4)\n",
    "result_cols_mismatch = pd.concat([df1, df4], axis=1, join=\"outer\")\n",
    "print(\"\\n--- 按列连接 df1 和 df4 (行索引不匹配) ---\")\n",
    "print(result_cols_mismatch)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8cda7f8-c29e-477c-863e-d94d277e9f0a",
   "metadata": {},
   "source": [
    "### 处理不匹配的轴"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5120f229-fd1f-49e3-9ff9-8743ebb1fe45",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_a = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})\n",
    "df_b = pd.DataFrame({'C': [5, 6], 'B': [7, 8]}) # 注意 df_b 也有 'B' 列，但没有 'A'\n",
    "\n",
    "# 默认 outer join (按列连接时，如果列名不完全相同)\n",
    "# 结果会包含所有列 'A', 'B', 'C'\n",
    "result_outer_join = pd.concat([df_a, df_b], axis=1)\n",
    "print(\"\\n--- concat (axis=1, join='outer') ---\")\n",
    "print(result_outer_join)\n",
    "\n",
    "# inner join (按列连接时，只保留公共列)\n",
    "# 结果只包含公共列 'B'\n",
    "result_inner_join = pd.concat([df_a, df_b], axis=1, join='inner')\n",
    "print(\"\\n--- concat (axis=1, join='inner') ---\")\n",
    "print(result_inner_join)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
